package com.eascs.spider.yuqing.worker.spider;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.eascs.spider.yuqing.common.proj.Platform;
import com.eascs.spider.yuqing.common.proj.Task;
import com.eascs.spider.yuqing.common.util.CommonUtil;
import com.eascs.spider.yuqing.common.util.Downloader;
import com.eascs.spider.yuqing.worker.util.MongodbUtil;

/**
 * 新农商网农村资讯
 * @author hzp
 *
 */
public class NcInfoSpider extends AnotherSpider{

	public NcInfoSpider(Task task) {
		super(task);
	}

	@Override
	public void run() {
		try {
			if(task.getHomepage().equals(Task.HOMEPAGE_YES)){
				cralwerHomepage();
			}else if(task.getUrl().matches("http://nc.mofcom.gov.cn/article/.*?.html")){
				crawlerArticle();
			}else{
				logger.warn("遇到其他的任务:{}" ,JSON.toJSONString(task));
			}
		} catch (Exception e) {
			logger.error("新农商网农业资讯爬取发生异常,url:{}" ,task.getUrl() ,e);
		}
	}

	//爬主页
	private void cralwerHomepage(){
		//1,从农业资讯首页开始爬
		String indexUrl = "http://nc.mofcom.gov.cn/channel/information/zxIndex.shtml";
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://nc.mofcom.gov.cn/");
		header.put("Host", "nc.mofcom.gov.cn");
		Downloader downloader =Downloader.build()
										  .setHeader(header)
										  .setHttpMethod("GET")
										  .setUrl(indexUrl)
										  .download();
		String html = downloader.getHtml();
		extractUrl(html);
		//2,首页加载更多,默认ajax加载10次
		String ajaxUrl = "http://nc.mofcom.gov.cn/nc/channel/information/getList.do";
		header.put("Referer", "http://nc.mofcom.gov.cn/channel/information/zxIndex.shtml");
		header.put("Origin", "http://nc.mofcom.gov.cn");
		header.put("X-Requested-With", "XMLHttpRequest");
		Map<String, String> param = new HashMap<>();
		param.put("action", "more_list");
		downloader.setHeader(header);
		downloader.setUrl(ajaxUrl);
		Pattern ajapa = Pattern.compile("\"redirectUrl\":\"(.*?)\"");
		for(int i=1;i<=10;i++){
			param.put("clickNum", i +"");
			
			String moreList = downloader.setParam(param)
										.setHttpMethod(Task.HTTP_POST)
										.download()
										.getHtml();
			Matcher tempMat = ajapa.matcher(moreList);
			while(tempMat.find()){
				String url = "http://nc.mofcom.gov.cn" + tempMat.group(1);
				Task task = new Task();
				task.setHeaderMap(header)
					.setHomepage(Task.HOMEPAGE_NO)
					.setMethod(Task.HTTP_GET)
					.setUrl(url);
				manager.addSendQueue(task);
			}
			CommonUtil.sleep(1000);
		}
		
		//3,农业要闻，不翻页
		String cateinfoUrl = "http://nc.mofcom.gov.cn/list/nyyw/1/cateinfo.html";
		header.put("Referer", "http://nc.mofcom.gov.cn/channel/information/zxIndex.shtml");
		header.remove("X-Requested-With");
		String cateinfoHtml = downloader.setUrl(cateinfoUrl)
										.setHttpMethod(Task.HTTP_GET)
										.setHeader(header)
										.setParam(null)
										.download()
										.getHtml();
		extractUrl(cateinfoHtml);
		//4,地方新闻，翻页，停止翻页以检测到非当天日期为准 yyyy-MM-dd
		String localNewsUrl = "http://nc.mofcom.gov.cn/channel/information/zxLocalNews.shtml";
		String localNewsHtml = downloader.setUrl(localNewsUrl)
										 .setHttpMethod(Task.HTTP_GET)
										 .setHeader(header)
										 .download()
										 .getHtml();
		
		extractUrl(localNewsHtml);
		for(int i=2;i<10;i++){
			header.put("Referer", downloader.getUrl());
			Map<String, String> paramTemp = new HashMap<>();
			paramTemp.put("page", i+"");
			String htmlTemp = downloader.setUrl(localNewsUrl)
										.setHttpMethod(Task.HTTP_GET)
										.setParam(paramTemp)
										.setHeader(header)
										.download()
										.getHtml();
			
			if(!checkIfTodayLocalNews(htmlTemp)){
				logger.info("地方新闻翻页检测到非今日的文章，中断翻页");
				break;
			}
			extractUrl(htmlTemp);
			CommonUtil.sleep(1000);
		}
		//5,市场分析，翻页，停止翻页以检测到非当天日期为准 yyyy-MM-dd,ajax请求
		String scfxUrl = "http://nc.mofcom.gov.cn/nc_cms/front/newcms/list";
		Map<String, String> headscfx = new HashMap<>();
		headscfx.put("Referer", "http://nc.mofcom.gov.cn/list/scfx/1/cateinfo.html");
		headscfx.put("Host", "nc.mofcom.gov.cn");
		headscfx.put("Origin", "http://nc.mofcom.gov.cn");
		headscfx.put("X-Requested-With", "XMLHttpRequest");
		Map<String, String> paramscfx = new HashMap<>();
		paramscfx.put("cid", "scfx");
		downloader.setHeader(headscfx).setUrl(scfxUrl).setHttpMethod("POST");
		for(int i=1;i<=5;i++){
			paramscfx.put("pageNumber", i+"");
			String scfxHtml = downloader.setParam(paramscfx).download().getHtml();
			if(!checkIfTodayscfxAndExtractUrl(scfxHtml))
				break;
			CommonUtil.sleep(1000);
		}		
	}
	
	//爬文章信息
	private void crawlerArticle() throws ParseException{
		Map<String,String> header = task.getHeaderMap();
		String html = downloader.setUrl(task.getUrl())
								.setHeader(header)
								.setHttpMethod(Task.HTTP_GET)
								.download()
								.getHtml();
		extractInfo(html);
	}
	
	
	//从中提取文章信息
	public void extractInfo(String html) throws ParseException{
		Matcher matcher = Pattern.compile("var (.*?) = '(.*?)';").matcher(html);
		Map<String, Object> data = new HashMap<String, Object>();
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		data.put("crawler_time", new Date(System.currentTimeMillis()));
		data.put("url", task.getUrl());
		while(matcher.find()){
			String name = matcher.group(1);
			String value = matcher.group(2);
			if(value.length() == 0)
				continue;
			if(name.equals("publishTime")){
				data.put(name, sdf.parse(value));
			}else{
				data.put(name, value);
			}
		}
		logger.info("新农商网农业资讯信息,单条数据,url:{}" ,task.getUrl());
		MongodbUtil.setYuqing(data, Platform.NC_INFO);
	}
	
	//从中提取文章链接
	public void extractUrl(String html){
		Matcher matcher = Pattern.compile("<a href=\"(/article/.*?)\"").matcher(html);
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://nc.mofcom.gov.cn/channel/information/zxIndex.shtml");
		header.put("Host", "nc.mofcom.gov.cn");
		
		while(matcher.find()){
			String url = "http://nc.mofcom.gov.cn" + matcher.group(1);
			Task task = new Task();
			task.setHeaderMap(header)
				.setHomepage(Task.HOMEPAGE_NO)
				.setMethod(Task.HTTP_GET)
				.setUrl(url);
			manager.addSendQueue(task);
		}
	}
	
	//检查是否有不是今天日期的文章，有则返回false----->农业资讯模块（地方新闻）
	public boolean checkIfTodayLocalNews(String html){
		Document document = Jsoup.parse(html);
		Elements ies = document.select("div#liList i");
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String today = sdf.format(new Date(System.currentTimeMillis()));
		for(Element i : ies){
			if(!today.equals(i.text()))
				return false;
		}
		return true;
	}
	
	//检查是否有不是今天日期的文章，有则返回false,同时提取url----->农业资讯模块（市场分析）
	private boolean checkIfTodayscfxAndExtractUrl(String html){
		Matcher matcher = Pattern.compile("\"publishTimeStr\":\"(.*?) (.*?)\",\"url\":\"(.*?)\"").matcher(html);
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String today = sdf.format(new Date(System.currentTimeMillis()));
		
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://nc.mofcom.gov.cn/list/scfx/1/cateinfo.html");
		header.put("Host", "nc.mofcom.gov.cn");
		while(matcher.find()){
			if(!today.equals(matcher.group(1))){
				logger.info("农业资讯市场分析，翻页中发现非今日发表的文章，中断翻页");
				return false;
			}
			String url = "http://nc.mofcom.gov.cn" + matcher.group(3);
			Task task = new Task();
			task.setHomepage(Task.HOMEPAGE_NO)
				.setUrl(url)
				.setMethod("GET")
				.setHeaderMap(header);
			manager.addSendQueue(task);
		}
		return true;
	}	
	
	public static void main(String[] args) throws ParseException{
		String this_year = Calendar.getInstance().get(Calendar.YEAR) +"";
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		Date date = sdf.parse(this_year +"-" +"12-22 15:30:36");
		System.out.println(date);
	}
}
